{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import scipy.optimize\n",
    "from keras.datasets import mnist\n",
    "from autoencoder import load_MNIST, sample_images, sparse_autoencoder, display_network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  allow your sparse autoencoder to get good filters; you do not need to\n",
    "#  change the parameters below.\n",
    "\n",
    "# number of input units\n",
    "visible_size = 28 * 28\n",
    "# number of input units\n",
    "hidden_size = 196\n",
    "\n",
    "# desired average activation of the hidden units.\n",
    "# (This was denoted by the Greek alphabet rho, which looks like a lower-case \"p\",\n",
    "#  in the lecture notes).\n",
    "sparsity_param = 0.1\n",
    "# weight decay parameter\n",
    "lambda_ = 3e-3\n",
    "# weight of sparsity penalty term\n",
    "beta = 3\n",
    "# debug\n",
    "debug = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  After implementing sampleIMAGES, the display_network command should\n",
    "#  display a random sample of 200 patches from the dataset\n",
    "\n",
    "# Loading Sample Images\n",
    "# patches = sample_images.sample_images()\n",
    "\n",
    "# Loading 10K images from MNIST database\n",
    "#(x_train, y_train), (x_test, y_test) = mnist.load_data(already_path='data/mnist.npz')\n",
    "#images = x_train.reshape(60000, 784).T\n",
    "images = load_MNIST.load_MNIST_images('data/mnist/train-images-idx3-ubyte')\n",
    "patches = images[:, 0:10000]\n",
    "\n",
    "#  Obtain random parameters theta\n",
    "theta = sparse_autoencoder.initialize(hidden_size, visible_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "321.75994418906583 [1.94896688e-04 4.65183778e-05 1.47222528e-04 ... 1.12329383e-01\n",
      " 1.18373929e-01 1.41804442e-01]\n"
     ]
    }
   ],
   "source": [
    "#  You can implement all of the components (squared error cost, weight decay term,\n",
    "#  sparsity penalty) in the cost function at once, but it may be easier to do\n",
    "#  it step-by-step and run gradient checking (see STEP 3) after each step.  We\n",
    "#  suggest implementing the sparseAutoencoderCost function using the following steps:\n",
    "#\n",
    "#  (a) Implement forward propagation in your neural network, and implement the\n",
    "#      squared error term of the cost function.  Implement backpropagation to\n",
    "#      compute the derivatives.   Then (using lambda=beta=0), run Gradient Checking\n",
    "#      to verify that the calculations corresponding to the squared error cost\n",
    "#      term are correct.\n",
    "#\n",
    "#  (b) Add in the weight decay term (in both the cost function and the derivative\n",
    "#      calculations), then re-run Gradient Checking to verify correctness.\n",
    "#\n",
    "#  (c) Add in the sparsity penalty term, then re-run Gradient Checking to\n",
    "#      verify correctness.\n",
    "#\n",
    "#  Feel free to change the training settings when debugging your\n",
    "#  code.  (For example, reducing the training set size or\n",
    "#  number of hidden units may make your code run faster; and setting beta\n",
    "#  and/or lambda to zero may be helpful for debugging.)  However, in your\n",
    "#  final submission of the visualized weights, please use parameters we\n",
    "#  gave in Step 0 above.\n",
    "\n",
    "(cost, grad) = sparse_autoencoder.sparse_autoencoder_cost(theta, visible_size,\n",
    "                                                          hidden_size, lambda_,\n",
    "                                                          sparsity_param, beta, patches)\n",
    "print(cost, grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def display(x):\n",
    "    global epoch\n",
    "    if epoch % 5 == 0:\n",
    "        opt_theta = x\n",
    "        W1 = opt_theta[0:hidden_size * visible_size].reshape(hidden_size, visible_size).transpose()\n",
    "        display_network.display_network(W1)\n",
    "    epoch+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      fun: 27.8881124268482\n",
      " hess_inv: <308308x308308 LbfgsInvHessProduct with dtype=float64>\n",
      "      jac: array([2.07792609e-04, 1.29370051e-05, 2.05711033e-04, ...,\n",
      "       2.28926109e-03, 2.93657274e-03, 2.80342113e-03])\n",
      "  message: b'STOP: TOTAL NO. of ITERATIONS REACHED LIMIT'\n",
      "     nfev: 23\n",
      "      nit: 20\n",
      "   status: 1\n",
      "  success: False\n",
      "        x: array([ 0.0692642 ,  0.00431234,  0.06857034, ..., -1.21832189,\n",
      "       -1.14451733, -1.15952624])\n"
     ]
    }
   ],
   "source": [
    "#  sparseAutoencoderCost is correct, You can start training your sparse\n",
    "#  autoencoder with minFunc (L-BFGS).\n",
    "\n",
    "#  Randomly initialize the parameters\n",
    "epoch = 0\n",
    "\n",
    "theta = sparse_autoencoder.initialize(hidden_size, visible_size)\n",
    "\n",
    "J = lambda x: sparse_autoencoder.sparse_autoencoder_cost(x, visible_size, hidden_size,\n",
    "                                                         lambda_, sparsity_param,\n",
    "                                                         beta, patches)\n",
    "options_ = {'maxiter': 20, 'disp': True}\n",
    "result = scipy.optimize.minimize(J, theta, method='L-BFGS-B', jac=True, options=options_, callback=display)\n",
    "\n",
    "\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "W1 = opt_theta[0:hidden_size * visible_size].reshape(hidden_size, visible_size).transpose()\n",
    "display_network.display_network(W1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(784, 196)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "W1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
